<!DOCTYPE html>
<html lang=zh>
<head><meta name="generator" content="Hexo 3.9.0">
    <!-- so meta -->
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="HandheldFriendly" content="True">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
    <meta name="description" content="前言在写 nodejs 爬虫的过程中，原网站可能会对某一时间段内集中访问该页面的 ip 进行封杀。那么如何动态设置每次爬取使用的 ip 地址以及浏览器头部信息呢？ 动态 userAgent这是我收集到的常用的浏览器头部信息，每次爬取的时候从中随机选取一个，并使用 superAgent 设置请求头部的 User-Agent 字段就好了。 userAgent.js1234567891011121314">
<meta name="keywords" content="nodejs,爬虫">
<meta property="og:type" content="article">
<meta property="og:title" content="nodejs爬虫如何设置动态ip以及userAgent">
<meta property="og:url" content="https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/index.html">
<meta property="og:site_name" content="月光倾城的代码小屋">
<meta property="og:description" content="前言在写 nodejs 爬虫的过程中，原网站可能会对某一时间段内集中访问该页面的 ip 进行封杀。那么如何动态设置每次爬取使用的 ip 地址以及浏览器头部信息呢？ 动态 userAgent这是我收集到的常用的浏览器头部信息，每次爬取的时候从中随机选取一个，并使用 superAgent 设置请求头部的 User-Agent 字段就好了。 userAgent.js1234567891011121314">
<meta property="og:locale" content="zh-CN">
<meta property="og:updated_time" content="2020-01-15T06:30:09.120Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="nodejs爬虫如何设置动态ip以及userAgent">
<meta name="twitter:description" content="前言在写 nodejs 爬虫的过程中，原网站可能会对某一时间段内集中访问该页面的 ip 进行封杀。那么如何动态设置每次爬取使用的 ip 地址以及浏览器头部信息呢？ 动态 userAgent这是我收集到的常用的浏览器头部信息，每次爬取的时候从中随机选取一个，并使用 superAgent 设置请求头部的 User-Agent 字段就好了。 userAgent.js1234567891011121314">
    
    
        
          
              <link rel="shortcut icon" href="/images/favicon.ico">
          
        
        
          
            <link rel="icon" type="image/png" href="/images/favicon-192x192.png" sizes="192x192">
          
        
        
          
            <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon.png">
          
        
    
    <!-- title -->
    <title>nodejs爬虫如何设置动态ip以及userAgent</title>
    <!-- styles -->
    <link rel="stylesheet" href="/css/style.css">
    <!-- persian styles -->
    
      <link rel="stylesheet" href="/css/rtl.css">
    
    <!-- rss -->
    
    
</head>

<body class="max-width mx-auto px3 ltr">
    
      <div id="header-post">
  <a id="menu-icon" href="#"><i class="fas fa-bars fa-lg"></i></a>
  <a id="menu-icon-tablet" href="#"><i class="fas fa-bars fa-lg"></i></a>
  <a id="top-icon-tablet" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');" style="display:none;"><i class="fas fa-chevron-up fa-lg"></i></a>
  <span id="menu">
    <span id="nav">
      <ul>
         
          <li><a href="/">首页</a></li>
         
          <li><a href="/archives/">归档</a></li>
         
          <li><a href="/search/">搜索</a></li>
         
          <li><a href="/tags/">标签</a></li>
         
          <li><a href="/others/">其他文章</a></li>
        
      </ul>
    </span>
    <br/>
    <span id="actions">
      <ul>
        
        <li><a class="icon" href="/2017/12/08/vue_notice_not_save/"><i class="fas fa-chevron-left" aria-hidden="true" onmouseover="$('#i-prev').toggle();" onmouseout="$('#i-prev').toggle();"></i></a></li>
        
        
        <li><a class="icon" href="/2017/12/05/nodejs_babel/"><i class="fas fa-chevron-right" aria-hidden="true" onmouseover="$('#i-next').toggle();" onmouseout="$('#i-next').toggle();"></i></a></li>
        
        <li><a class="icon" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');"><i class="fas fa-chevron-up" aria-hidden="true" onmouseover="$('#i-top').toggle();" onmouseout="$('#i-top').toggle();"></i></a></li>
        <li><a class="icon" href="#"><i class="fas fa-share-alt" aria-hidden="true" onmouseover="$('#i-share').toggle();" onmouseout="$('#i-share').toggle();" onclick="$('#share').toggle();return false;"></i></a></li>
      </ul>
      <span id="i-prev" class="info" style="display:none;">上一篇</span>
      <span id="i-next" class="info" style="display:none;">下一篇</span>
      <span id="i-top" class="info" style="display:none;">返回顶部</span>
      <span id="i-share" class="info" style="display:none;">分享文章</span>
    </span>
    <br/>
    <div id="share" style="display: none">
      <ul>
  <li><a class="icon" href="http://www.facebook.com/sharer.php?u=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/"><i class="fab fa-facebook " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://twitter.com/share?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&text=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-twitter " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.linkedin.com/shareArticle?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-linkedin " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://pinterest.com/pin/create/bookmarklet/?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&is_video=false&description=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-pinterest " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="mailto:?subject=nodejs爬虫如何设置动态ip以及userAgent&body=Check out this article: https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/"><i class="fas fa-envelope " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://getpocket.com/save?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-get-pocket " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://reddit.com/submit?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-reddit " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.stumbleupon.com/submit?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-stumbleupon " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://digg.com/submit?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-digg " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.tumblr.com/share/link?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&name=nodejs爬虫如何设置动态ip以及userAgent&description="><i class="fab fa-tumblr " aria-hidden="true"></i></a></li>
</ul>

    </div>
    <div id="toc">
      <ol class="toc"><li class="toc-item toc-level-4"><a class="toc-link" href="#前言"><span class="toc-number">1.</span> <span class="toc-text">前言</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#动态-userAgent"><span class="toc-number">2.</span> <span class="toc-text">动态 userAgent</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#userAgent-js"><span class="toc-number">3.</span> <span class="toc-text">userAgent.js</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#app-js"><span class="toc-number">4.</span> <span class="toc-text">app.js</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#动态-ip"><span class="toc-number">5.</span> <span class="toc-text">动态 ip</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#package-json"><span class="toc-number">6.</span> <span class="toc-text">package.json</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#app-js-1"><span class="toc-number">7.</span> <span class="toc-text">app.js</span></a></li></ol>
    </div>
  </span>
</div>

    
    <div class="content index py4">
        
        <article class="post" itemscope itemtype="http://schema.org/BlogPosting">
  <header>
    
    <h1 class="posttitle" itemprop="name headline">
        nodejs爬虫如何设置动态ip以及userAgent
    </h1>



    <div class="meta">
      <span class="author" itemprop="author" itemscope itemtype="http://schema.org/Person">
        <span itemprop="name">月光倾城的代码小屋</span>
      </span>
      
    <div class="postdate">
        <time datetime="2017-12-05T17:45:31.000Z" itemprop="datePublished">2017-12-05</time>
    </div>


      

      
    <div class="article-tag">
        <i class="fas fa-tag"></i>
        <a class="tag-link" href="/tags/nodejs/">nodejs</a>, <a class="tag-link" href="/tags/爬虫/">爬虫</a>
    </div>


    </div>
  </header>
  

  <div class="content" itemprop="articleBody">
    <h4 id="前言"><a href="#前言" class="headerlink" title="前言"></a>前言</h4><p>在写 nodejs 爬虫的过程中，原网站可能会对某一时间段内集中访问该页面的 ip 进行封杀。那么如何动态设置每次爬取使用的 ip 地址以及浏览器头部信息呢？</p>
<h4 id="动态-userAgent"><a href="#动态-userAgent" class="headerlink" title="动态 userAgent"></a>动态 userAgent</h4><p>这是我收集到的常用的浏览器头部信息，每次爬取的时候从中随机选取一个，并使用 superAgent 设置请求头部的 User-Agent 字段就好了。</p>
<h4 id="userAgent-js"><a href="#userAgent-js" class="headerlink" title="userAgent.js"></a><code>userAgent.js</code></h4><figure class="highlight js"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">const</span> userAgents = [</span><br><span class="line">  <span class="string">'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12'</span>,</span><br><span class="line">  <span class="string">'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0) ,Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9'</span>,</span><br><span class="line">  <span class="string">'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)'</span>,</span><br><span class="line">  <span class="string">'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre'</span>,</span><br><span class="line">  <span class="string">'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)'</span>,</span><br><span class="line">  <span class="string">'Opera/9.25 (Windows NT 5.1; U; en), Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9'</span>,</span><br><span class="line">  <span class="string">'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'</span></span><br><span class="line">];</span><br><span class="line"></span><br><span class="line"><span class="built_in">module</span>.exports = userAgents;</span><br></pre></td></tr></table></figure>
<h4 id="app-js"><a href="#app-js" class="headerlink" title="app.js"></a><code>app.js</code></h4><figure class="highlight js"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> request <span class="keyword">from</span> <span class="string">'superagent'</span>;</span><br><span class="line"><span class="keyword">import</span> userAgents <span class="keyword">from</span> <span class="string">'../src/userAgent'</span>;</span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">function</span> <span class="title">doRequest</span>(<span class="params"></span>) </span>&#123;</span><br><span class="line">  <span class="keyword">let</span> userAgent = userAgents[<span class="built_in">parseInt</span>(<span class="built_in">Math</span>.random() * userAgents.length)];</span><br><span class="line">  request</span><br><span class="line">    .get(<span class="string">'http://www.xxx.com'</span>)</span><br><span class="line">    .set(&#123; <span class="string">'User-Agent'</span>: userAgent &#125;)</span><br><span class="line">    .timeout(&#123; <span class="attr">response</span>: <span class="number">5000</span>, <span class="attr">deadline</span>: <span class="number">60000</span> &#125;)</span><br><span class="line">    .end(<span class="keyword">async</span> (err, res) =&gt; &#123;</span><br><span class="line">      <span class="comment">// 处理数据</span></span><br><span class="line">    &#125;);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h4 id="动态-ip"><a href="#动态-ip" class="headerlink" title="动态 ip"></a>动态 ip</h4><p>设置动态 IP 需要用到一个 superagent 插件—<a href="https://github.com/TooTallNate/superagent-proxy" target="_blank" rel="noopener">superagent-proxy</a>，除此之外为了避免每次爬取时都去获取一次动态 IP 的列表，我将爬取到的动态 IP 列表存放在 redis 中，并设置 10 分钟的过期时间。数据过期之后再重新发送获取动态 IP 的请求。<br>ps: 这里我使用的动态 IP 是<a href="http://www.pcdaili.com/" target="_blank" rel="noopener">爬虫网络科技公司</a>提供的免费代理，因为免费所以难免会有些缺陷。有时候使用他的代理 ip 并不能访问得通，我在后面会做单独的处理。</p>
<h4 id="package-json"><a href="#package-json" class="headerlink" title="package.json"></a><code>package.json</code></h4><figure class="highlight json"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"name"</span>: <span class="string">"xxx"</span>,</span><br><span class="line">  <span class="attr">"version"</span>: <span class="string">"1.0.0"</span>,</span><br><span class="line">  <span class="attr">"description"</span>: <span class="string">"xxx"</span>,</span><br><span class="line">  <span class="attr">"main"</span>: <span class="string">"arf.js"</span>,</span><br><span class="line">  <span class="attr">"scripts"</span>: &#123;</span><br><span class="line">    <span class="attr">"arf"</span>: <span class="string">"nodemon src/app.js --exec babel-node --config package.json"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  <span class="attr">"keywords"</span>: [<span class="string">"爬虫"</span>],</span><br><span class="line">  <span class="attr">"author"</span>: <span class="string">"lidikang"</span>,</span><br><span class="line">  <span class="attr">"license"</span>: <span class="string">"MIT"</span>,</span><br><span class="line">  <span class="attr">"dependencies"</span>: &#123;</span><br><span class="line">    <span class="attr">"bluebird"</span>: <span class="string">"^3.5.1"</span>,</span><br><span class="line">    <span class="attr">"cheerio"</span>: <span class="string">"^1.0.0-rc.2"</span>,</span><br><span class="line">    <span class="attr">"eventproxy"</span>: <span class="string">"^1.0.0"</span>,</span><br><span class="line">    <span class="attr">"mongoose"</span>: <span class="string">"^4.13.6"</span>,</span><br><span class="line">    <span class="attr">"mongoose-findorcreate"</span>: <span class="string">"^2.0.0"</span>,</span><br><span class="line">    <span class="attr">"progress"</span>: <span class="string">"^2.0.0"</span>,</span><br><span class="line">    <span class="attr">"redis"</span>: <span class="string">"^2.8.0"</span>,</span><br><span class="line">    <span class="attr">"superagent"</span>: <span class="string">"^3.8.1"</span>,</span><br><span class="line">    <span class="attr">"superagent-proxy"</span>: <span class="string">"^1.0.2"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  <span class="attr">"devDependencies"</span>: &#123;</span><br><span class="line">    <span class="attr">"babel-cli"</span>: <span class="string">"^6.26.0"</span>,</span><br><span class="line">    <span class="attr">"babel-preset-es2015"</span>: <span class="string">"^6.24.1"</span>,</span><br><span class="line">    <span class="attr">"babel-preset-stage-2"</span>: <span class="string">"^6.24.1"</span>,</span><br><span class="line">    <span class="attr">"nodemon"</span>: <span class="string">"^1.12.4"</span></span><br><span class="line">  &#125;,</span><br><span class="line">  <span class="attr">"nodemonConfig"</span>: &#123;</span><br><span class="line">    <span class="attr">"ignore"</span>: [<span class="string">"ips.json"</span>, <span class="string">"docs/*"</span>],</span><br><span class="line">    <span class="attr">"delay"</span>: <span class="string">"2500"</span></span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<h4 id="app-js-1"><a href="#app-js-1" class="headerlink" title="app.js"></a><code>app.js</code></h4><figure class="highlight js"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> request <span class="keyword">from</span> <span class="string">'superagent'</span>;</span><br><span class="line"><span class="keyword">import</span> requestProxy <span class="keyword">from</span> <span class="string">'superagent-proxy'</span>;</span><br><span class="line"><span class="keyword">import</span> redis <span class="keyword">from</span> <span class="string">'redis'</span>;</span><br><span class="line"><span class="comment">// superagent添加使用代理ip的插件</span></span><br><span class="line">requestProxy(request);</span><br><span class="line"><span class="comment">// redis promise化</span></span><br><span class="line">bluebird.promisifyAll(redis.RedisClient.prototype);</span><br><span class="line">bluebird.promisifyAll(redis.Multi.prototype);</span><br><span class="line"><span class="comment">// 建立mongoose和redis连接</span></span><br><span class="line"><span class="keyword">const</span> redisClient = connectRedis();</span><br><span class="line"></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * 初始化redis</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="function"><span class="keyword">function</span> <span class="title">connectRedis</span>(<span class="params"></span>) </span>&#123;</span><br><span class="line">  <span class="keyword">let</span> client = redis.createClient(config.REDIS_URL);</span><br><span class="line">  client.on(<span class="string">'ready'</span>, <span class="function"><span class="keyword">function</span>(<span class="params">err</span>) </span>&#123;</span><br><span class="line">    <span class="built_in">console</span>.log(<span class="string">'redis连接 √'</span>);</span><br><span class="line">  &#125;);</span><br><span class="line">  client.on(<span class="string">'error'</span>, <span class="function"><span class="keyword">function</span>(<span class="params">err</span>) </span>&#123;</span><br><span class="line">    <span class="built_in">console</span>.log(<span class="string">`redis错误，<span class="subst">$&#123;err&#125;</span>  ×`</span>);</span><br><span class="line">  &#125;);</span><br><span class="line">  <span class="keyword">return</span> client;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="comment">/**</span></span><br><span class="line"><span class="comment"> * 请求免费代理，读取redis，如果代理信息已经过期，重新请求免费代理请求</span></span><br><span class="line"><span class="comment"> */</span></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">function</span> <span class="title">getProxyIp</span>(<span class="params"></span>) </span>&#123;</span><br><span class="line">  <span class="comment">// 先从redis读取缓存ip</span></span><br><span class="line">  <span class="keyword">let</span> localIpStr = <span class="keyword">await</span> redisClient.getAsync(<span class="string">'proxy_ips'</span>);</span><br><span class="line">  <span class="keyword">let</span> ips = <span class="literal">null</span>;</span><br><span class="line">  <span class="comment">// 如果本地存在，则随机返回其中一个ip，否则重新请求</span></span><br><span class="line">  <span class="keyword">if</span> (localIpStr) &#123;</span><br><span class="line">    <span class="keyword">let</span> localIps = localIpStr.split(<span class="string">','</span>);</span><br><span class="line">    <span class="keyword">return</span> localIps[<span class="built_in">parseInt</span>(<span class="built_in">Math</span>.random() * localIps.length)];</span><br><span class="line">  &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">    <span class="keyword">let</span> ipsJson = (<span class="keyword">await</span> request.get(<span class="string">'http://api.pcdaili.com/?orderid=888888888&amp;num=100&amp;protocol=1&amp;method=1&amp;an_ha=1&amp;sp1=1&amp;sp2=1&amp;format=json&amp;sep=1'</span>)).body;</span><br><span class="line">    <span class="keyword">let</span> isRequestSuccess = <span class="literal">false</span>;</span><br><span class="line">    <span class="keyword">if</span> (ipsJson &amp;&amp; ipsJson.data.proxy_list) &#123;</span><br><span class="line">      ips = ipsJson.data.proxy_list;</span><br><span class="line">      isRequestSuccess = <span class="literal">true</span>;</span><br><span class="line">    &#125; <span class="keyword">else</span> &#123;</span><br><span class="line">      ips = [<span class="string">'http://127.0.0.1'</span>];</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment">// 将爬取结果存入本地，缓存时间10分钟</span></span><br><span class="line">    <span class="keyword">if</span> (isRequestSuccess) &#123;</span><br><span class="line">      redisClient.set(<span class="string">'proxy_ips'</span>, ips.join(<span class="string">','</span>), <span class="string">'EX'</span>, <span class="number">10</span> * <span class="number">60</span>);</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="keyword">return</span> ips[<span class="built_in">parseInt</span>(<span class="built_in">Math</span>.random() * ips.length)];</span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">function</span> <span class="title">doRequest</span>(<span class="params"></span>) </span>&#123;</span><br><span class="line">  <span class="keyword">let</span> userAgent = userAgents[<span class="built_in">parseInt</span>(<span class="built_in">Math</span>.random() * userAgents.length)];</span><br><span class="line">  <span class="keyword">let</span> ip = <span class="keyword">await</span> getProxyIp();</span><br><span class="line">  <span class="keyword">let</span> useIp = <span class="string">'http://'</span> + ip;</span><br><span class="line">  request</span><br><span class="line">    .get(<span class="string">'http://www.xxx.com'</span>)</span><br><span class="line">    .set(&#123; <span class="string">'User-Agent'</span>: userAgent &#125;)</span><br><span class="line">    .timeout(&#123; <span class="attr">response</span>: <span class="number">5000</span>, <span class="attr">deadline</span>: <span class="number">60000</span> &#125;)</span><br><span class="line">    .proxy(ip)</span><br><span class="line">    .end(<span class="keyword">async</span> (err, res) =&gt; &#123;</span><br><span class="line">      <span class="comment">// 处理数据</span></span><br><span class="line">    &#125;);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>之前说爬虫网络科技的免费 ip 有些缺陷—代理成功率有些低。这点必须想办法去修复，原理其实很简单，既然一次不成功那我就换个 IP 再试，直到成功了我才去开始执行解析 html 的逻辑</p>
<figure class="highlight js"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">function</span> <span class="title">doRequest</span>(<span class="params"></span>) </span>&#123;</span><br><span class="line">  <span class="keyword">let</span> userAgent = userAgents[<span class="built_in">parseInt</span>(<span class="built_in">Math</span>.random() * userAgents.length)];</span><br><span class="line">  <span class="keyword">let</span> ip = <span class="keyword">await</span> getProxyIp();</span><br><span class="line">  <span class="keyword">let</span> useIp = <span class="string">'http://'</span> + ip;</span><br><span class="line">  request</span><br><span class="line">    .get(<span class="string">'http://www.xxx.com'</span>)</span><br><span class="line">    .set(&#123; <span class="string">'User-Agent'</span>: userAgent &#125;)</span><br><span class="line">    .timeout(&#123; <span class="attr">response</span>: <span class="number">5000</span>, <span class="attr">deadline</span>: <span class="number">60000</span> &#125;)</span><br><span class="line">    .proxy(ip)</span><br><span class="line">    .end(<span class="keyword">async</span> (err, res) =&gt; &#123;</span><br><span class="line">      <span class="keyword">if</span> (err) &#123;</span><br><span class="line">        <span class="built_in">console</span>.log(<span class="string">`爬取页面失败，<span class="subst">$&#123;err&#125;</span>，正在重新寻找代理ip... ×`</span>);</span><br><span class="line">        <span class="comment">// 如果是代理ip无法访问，另外选择一个代理</span></span><br><span class="line">        doRequest(<span class="string">'http://'</span> + (<span class="keyword">await</span> getProxyIp()), userAgents[<span class="built_in">parseInt</span>(<span class="built_in">Math</span>.random() * userAgents.length)]);</span><br><span class="line">        <span class="keyword">return</span>;</span><br><span class="line">      &#125;</span><br><span class="line">      <span class="comment">// 解析html</span></span><br><span class="line">      <span class="built_in">console</span>.log(<span class="string">'爬取页面  √'</span>);</span><br><span class="line">      <span class="keyword">await</span> parseDivision(res.text);</span><br><span class="line">    &#125;);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>原文请查看<a href="http://andyliwr.github.io/2017/12/05/nodejs_spider_ip/">http://andyliwr.github.io/2017/12/05/nodejs_spider_ip/</a></p>

  </div>
  <div class="statement">
    <hr>  
    1. 如果本文帮到了您，不妨点一下右上角的”<i class="fas fa-share-alt"></i>“按钮。<br>
    2. 除非注明，本博文章均为原创，转载请以链接形式标明本文地址。<br>
    3. 如果你有任何疑问，欢迎发送邮件到我的邮箱<a href="mailto:andyliwr@outlook.com">andyliwr@outlook.com<br>
  </div>
</article>


    <div class="blog-post-comments">
        <div id="gitalk_thread">
            <noscript>加载评论需要在浏览器启用 JavaScript 脚本支持。</noscript>
        </div>
    </div>



        
          <div id="footer-post-container">
  <div id="footer-post">

    <div id="nav-footer" style="display: none">
      <ul>
         
          <li><a href="/">首页</a></li>
         
          <li><a href="/archives/">归档</a></li>
         
          <li><a href="/search/">搜索</a></li>
         
          <li><a href="/tags/">标签</a></li>
         
          <li><a href="/others/">其他文章</a></li>
        
      </ul>
    </div>

    <div id="toc-footer" style="display: none">
      <ol class="toc"><li class="toc-item toc-level-4"><a class="toc-link" href="#前言"><span class="toc-number">1.</span> <span class="toc-text">前言</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#动态-userAgent"><span class="toc-number">2.</span> <span class="toc-text">动态 userAgent</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#userAgent-js"><span class="toc-number">3.</span> <span class="toc-text">userAgent.js</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#app-js"><span class="toc-number">4.</span> <span class="toc-text">app.js</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#动态-ip"><span class="toc-number">5.</span> <span class="toc-text">动态 ip</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#package-json"><span class="toc-number">6.</span> <span class="toc-text">package.json</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#app-js-1"><span class="toc-number">7.</span> <span class="toc-text">app.js</span></a></li></ol>
    </div>

    <div id="share-footer" style="display: none">
      <ul>
  <li><a class="icon" href="http://www.facebook.com/sharer.php?u=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/"><i class="fab fa-facebook fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://twitter.com/share?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&text=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-twitter fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.linkedin.com/shareArticle?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-linkedin fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://pinterest.com/pin/create/bookmarklet/?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&is_video=false&description=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-pinterest fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="mailto:?subject=nodejs爬虫如何设置动态ip以及userAgent&body=Check out this article: https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/"><i class="fas fa-envelope fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://getpocket.com/save?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-get-pocket fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://reddit.com/submit?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-reddit fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.stumbleupon.com/submit?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-stumbleupon fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://digg.com/submit?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&title=nodejs爬虫如何设置动态ip以及userAgent"><i class="fab fa-digg fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.tumblr.com/share/link?url=https://andyliwr.github.io/2017/12/05/nodejs_spider_ip/&name=nodejs爬虫如何设置动态ip以及userAgent&description="><i class="fab fa-tumblr fa-lg" aria-hidden="true"></i></a></li>
</ul>

    </div>

    <div id="actions-footer">
        <a id="menu" class="icon" href="#" onclick="$('#nav-footer').toggle();return false;"><i class="fas fa-bars fa-lg" aria-hidden="true"></i> 菜单</a>
        <a id="toc" class="icon" href="#" onclick="$('#toc-footer').toggle();return false;"><i class="fas fa-list fa-lg" aria-hidden="true"></i> 目录</a>
        <a id="share" class="icon" href="#" onclick="$('#share-footer').toggle();return false;"><i class="fas fa-share-alt fa-lg" aria-hidden="true"></i> 分享</a>
        <a id="top" style="display:none" class="icon" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');"><i class="fas fa-chevron-up fa-lg" aria-hidden="true"></i> 返回顶部</a>
    </div>

  </div>
</div>

        
        <footer id="footer">
  <div class="footer-left">
    Copyright &copy; 2020 Dikang Li
  </div>
  <div class="footer-right">
    <nav>
      <ul>
         
          <li><a href="/">首页</a></li>
         
          <li><a href="/archives/">归档</a></li>
         
          <li><a href="/search/">搜索</a></li>
         
          <li><a href="/tags/">标签</a></li>
         
          <li><a href="/others/">其他文章</a></li>
        
      </ul>
    </nav>
  </div>
</footer>

    </div>
</body>
</html>
<!-- styles -->
<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
<link rel="stylesheet" href="/lib/justified-gallery/css/justifiedGallery.min.css">
<link rel="stylesheet" href="/lib/fancybox/jquery.fancybox.css">

<link rel="stylesheet" href="https://file.lantingshucheng.com/blog/v3/static/styles/gitalk.css">


<!-- jquery -->
<script src="/lib/jquery/jquery.min.js"></script>
<script src="/lib/justified-gallery/js/jquery.justifiedGallery.min.js"></script>
<script src="/lib/fancybox/jquery.fancybox.pack.js"></script>

<script type="text/javascript" src="https://file.lantingshucheng.com/blog/v3/static/scripts/md5.js"></script>
<script type="text/javascript" src="https://file.lantingshucheng.com/blog/v3/static/scripts/gitalk.min.js"></script>

<script src="/js/main.js"></script>
<!-- search -->

<!-- Google Analytics -->

<!-- Baidu Analytics -->

    <script type="text/javascript">
        var _hmt = _hmt || [];
        (function() {
            var hm = document.createElement("script");
            hm.src = "https://hm.baidu.com/hm.js?7851ca7a86be957be994acb77639a5be";
            var s = document.getElementsByTagName("script")[0];
            s.parentNode.insertBefore(hm, s);
        })();
    </script>

<!-- Disqus Comments -->


  <script type="text/javascript">
    window.onload = function(){
      if (window.location.href.indexOf('andyliwr.github.io') > -1) {
        var gitalk = new Gitalk({
          clientID: 'ad6f70df0c70d2b715f9',
          clientSecret: '40384bee244c1e17dd7c37733d21e9ef669e969c',
          repo: 'Andyliwr.github.io',
          owner: 'Andyliwr',
          admin: ['Andyliwr'],
          id: md5(location.pathname),
          distractionFreeMode: 'true'
        })
        gitalk.render('gitalk_thread')
      }
    }
  </script>


